# -*- coding: utf-8 -*-
import scrapy


class ShenmaSpider(scrapy.Spider):
    name = 'shenma'
    allowed_domains = []
    start_urls = []

    def __init__(self):
        for line in open("1.txt", "r", newline=None, encoding="utf-8"):
            self.start_urls.append("http://m.sm.cn/s?q=" + line)

    def start_requests(self):
        for url in self.start_urls:
            headers = {'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1'}
            yield scrapy.Request(url, headers=headers)

    def parse(self, response):
        print(response.request.headers['User-Agent'])
        textlist = response.xpath('//aside[@id="sider"]/div[@class="sider-cards"]/div[@class="sider-card relative-keywords"]/ul/li//span/text()').extract()

        for texts in textlist:
            print(texts)
            fo = open("text.txt", "a", newline=None, encoding="utf-8")
            if texts != "":
                fo.write(texts + "\n")
